package uk.org.mygrid.eobesity.csvserver.routes;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
//import java.io.StringReader;
//import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.configuration.Configuration;
//import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
//import org.restlet.data.Form;
import org.restlet.data.MediaType;
//import org.restlet.data.Parameter;
import org.restlet.data.Status;
import org.restlet.ext.fileupload.RestletFileUpload;
import org.restlet.ext.xml.DomRepresentation;
import org.restlet.representation.Representation;
//import org.restlet.representation.StringRepresentation;
import org.restlet.resource.Post;
import org.restlet.resource.ResourceException;
import org.restlet.resource.ServerResource;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
//import org.xml.sax.InputSource;
//import org.xml.sax.SAXException;

import uk.org.mygrid.eobesity.utils.CSVUtils;
import uk.org.mygrid.eobesity.utils.RServeUtil;

/**
 * Create a Zip file with CSV data for each of the datasets and variables
 * requested. The request is in the style of: <request> <Datasets> <Dataset
 * name="HSE2002"> <Variables>
 * <Variable>age</Variable><Variable>year</Variable><
 * Variable>dobmonth</Variable> </Variables> </Dataset> <Dataset name="HSE2003">
 * <Variables> <Variable>height</Variable> </Variables> </Dataset> <Dataset
 * year="HSE2006"> <Variables> <Variable>bmi</Variable> </Variables> </Dataset>
 * </Datasets> </request>
 * 
 * 
 * @author Ian Dunlop
 * 
 */
public class DownloadFile extends ServerResource {

    /** Date used in the name of the zip file */
    public static final String DATE_FORMAT_NOW = "yyyy-MM-dd HH:mm:ss"; //$NON-NLS-1$
    //queue to hold the csv jobs, fixed at 5 jobs processing at one time
    private static ExecutorService executor = Executors.newFixedThreadPool(5);
//    private static Map<String, String> numberProcessed = new HashMap<String, String>();
    private static List<String> failedJobs = new ArrayList<String>();
    private static Logger logger = Logger.getLogger(DownloadFile.class);
    private Configuration config;

    /**
     * Get the values sent in the request, parse for the concepts and the
     * variables required in the returned CSV files. Return the CSVs inside a
     * zip, one file for each dataset requested
     */
    @Post("xml")
    public Representation createAndDownloadArchive()
            throws ResourceException {
        DomRepresentation representation = null;
        try {
            try {
                setConfig(new PropertiesConfiguration("csv.properties"));
            } catch (Exception e) {
                logger.error("Problem with service properties", e);

            }

            // FileRepresentation fileRep = new FileRepresentation(file,
            // MediaType.TEXT_PLAIN, 12345);

            representation = new DomRepresentation(
                    MediaType.TEXT_XML);
            Document d = representation.getDocument();
            Element jobIDElement = d.createElement("jobid");
            final String jobID = UUID.randomUUID().toString();
            jobIDElement.appendChild(d.createTextNode(jobID));
            d.appendChild(jobIDElement);
            d.normalizeDocument();
//			getResponse().setEntity(representation);
            //write out the request for reference and due to problems with the stream not closing

            Representation requestEntity = getRequestEntity();

            // run the csv creation in a separate thread
            //add the job to the thread pool executor
            
            getExecutor().execute(createNewThread(jobID, requestEntity));

            setStatus(Status.SUCCESS_OK);

        } catch (FileNotFoundException e) {
            setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
        } catch (IOException e) {
            setStatus(Status.SERVER_ERROR_INTERNAL);
        }
        return representation;
    }

//    public static synchronized Map<String, String> getNumberProcessed() {
//        return numberProcessed;
//    }

    private static synchronized ExecutorService getExecutor() {
        return executor;
    }

    private synchronized void writeJobState() {
        String serializeDirectory = getConfig().getString("SerializeMap.0");

        File file = new File(serializeDirectory + "jobMap");
        try {
            FileOutputStream stream = new FileOutputStream(file);
            ObjectOutputStream objectStream = new ObjectOutputStream(stream);
            objectStream.writeObject(BaseResource.getCompletedJobList());
            objectStream.close();
        } catch (FileNotFoundException e) {
            logger.warn("Problem serializing job map", e);
        } catch (IOException e) {
            logger.warn("Problem serializing job map", e);
        }
    }
    
    private synchronized void writeSPSSState() {
        String serializeDirectory = getConfig().getString("SerializeMap.0");

        File file = new File(serializeDirectory + "spssJobMap");
        try {
            FileOutputStream stream = new FileOutputStream(file);
            ObjectOutputStream objectStream = new ObjectOutputStream(stream);
            objectStream.writeObject(BaseResource.getCompletedJobList());
            objectStream.close();
        } catch (FileNotFoundException e) {
            logger.warn("Problem serializing job map", e);
        } catch (IOException e) {
            logger.warn("Problem serializing job map", e);
        }
    }
    
//    private synchronized void writeStataState() {
//        String serializeDirectory = getConfig().getString("SerializeMap.0");
//
//        File file = new File(serializeDirectory + "stataJobMap");
//        try {
//            FileOutputStream stream = new FileOutputStream(file);
//            ObjectOutputStream objectStream = new ObjectOutputStream(stream);
//            objectStream.writeObject(BaseResource.getCompletedJobList());
//            objectStream.close();
//        } catch (FileNotFoundException e) {
//            logger.warn("Problem serializing job map", e);
//        } catch (IOException e) {
//            logger.warn("Problem serializing job map", e);
//        }
//    }

//    private void writeDatasetMap(String uuid,
//            Map<String, List<String>> datasetToVariableMap) {
//        String serializeDirectory = getConfig().getString("SerializeMap.0");
//        File file = new File(serializeDirectory + uuid + ".variableMap");
//        try {
//            FileOutputStream stream = new FileOutputStream(file);
//            ObjectOutputStream objectStream = new ObjectOutputStream(stream);
//            objectStream.writeObject(datasetToVariableMap);
//            objectStream.close();
//        } catch (FileNotFoundException e) {
//            logger.warn("Problem serializing dataset map");
//        } catch (IOException e) {
//            logger.warn("Problem serializing dataset map");
//        }
//    }

    public Runnable createNewThread(final String jobID, final Representation requestEntity) {
        Runnable runnable = new Runnable() {

            public void run() {
                try {
                    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
                	 DocumentBuilder parser = null;
                     try {
                         parser = factory.newDocumentBuilder();
                     } catch (ParserConfigurationException e) {
                         logger.error("Problem with thread for job " + jobID
                                 + ": " + e);
                     }
                     Document xmlDoc = null;
            		if (MediaType.MULTIPART_FORM_DATA.equals(getRequestEntity()
            				.getMediaType(), true)) {
            			DiskFileItemFactory diskFileFactory = new DiskFileItemFactory();
            			diskFileFactory.setSizeThreshold(1000240);

                        RestletFileUpload upload = new RestletFileUpload(diskFileFactory);
                        List<FileItem> items;
                        try {
                            items = upload.parseRequest(getRequest());
                            boolean found = false;
                            for (final Iterator<FileItem> it = items.iterator(); it
                                    .hasNext()
                                    && !found;) {
                                FileItem fi = it.next();
                                if (fi.getFieldName().equals("datasetrequest")) {
                                    found = true;
                                    xmlDoc = parser.parse(fi.getInputStream());
                                }
                            }
                        } catch (Exception e) {
                        	logger.error("Problem with thread for job " + jobID
                                    + ": " + e);
                        }

            		}
                    
                    BaseResource.getCompletedJobList().put(jobID, false);
                    writeJobState();
                    //what are all the required variables for the dataset
                    Map<String, List<String>> datasetToVariableMap = new HashMap<String, List<String>>();
                    //What is the survey type and year for the dataset
                    Map<String, Map<String,String>> datasetToInformationMap = new HashMap<String, Map<String,String>>();

                    XPathFactory xpFactory = XPathFactory.newInstance();
                    XPath xpath = xpFactory.newXPath();

                    String query = "//Email"; //$NON-NLS-1$
                    XPathExpression expr = null;
                    try {
                        expr = xpath.compile(query);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);
                    }

                    NodeList mailNodes = null;
                    try {
                        mailNodes = (NodeList) expr.evaluate(xmlDoc,
                                XPathConstants.NODESET);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);
                    }

                    String emailAddress = mailNodes.item(0).getFirstChild().getNodeValue();

                    query = "//Filename";
                    try {
                        expr = xpath.compile(query);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);

                    }

                    NodeList filenameNodes = null;
                    try {
                        filenameNodes = (NodeList) expr.evaluate(xmlDoc,
                                XPathConstants.NODESET);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);
                    }

                    String fileName = filenameNodes.item(0).getFirstChild().getNodeValue();

                    query = "//Dataset"; //$NON-NLS-1$
                    expr = null;
                    try {
                        expr = xpath.compile(query);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);

                    }
                    NodeList conceptNodes = null;
                    try {
                        conceptNodes = (NodeList) expr.evaluate(xmlDoc,
                                XPathConstants.NODESET);
                    } catch (XPathExpressionException e) {
                        logger.error("Problem with thread for job " + jobID
                                + ": " + e);
                    }

                    for (int m = 0; m < conceptNodes.getLength(); m++) {
                        Node conceptNode = conceptNodes.item(m);
                        NamedNodeMap attributes = conceptNode.getAttributes();

                        // year is actually the dataset name
                        Node datasetName = attributes.getNamedItem("name"); //$NON-NLS-1$
                        // String name = nameItem.getNodeValue();
                        String dataset = datasetName.getNodeValue();
                        
                        Node surveyType = attributes.getNamedItem("survey"); //$NON-NLS-1$
                        // String name = nameItem.getNodeValue();
                        String survey = surveyType.getNodeValue();
                        
                        Node surveyYear = attributes.getNamedItem("year"); //$NON-NLS-1$
                        // String name = nameItem.getNodeValue();
                        String year = surveyYear.getNodeValue();
                        
                        Node actualDatasetNameNode = attributes.getNamedItem("datasetname"); //$NON-NLS-1$
                        // String name = nameItem.getNodeValue();
                        String actualDatasetName = actualDatasetNameNode.getNodeValue();
                        
                        List<String> variableList = new ArrayList<String>();
                        // check if the dataset exists in the map
                        if (!datasetToVariableMap.containsKey(dataset)) {
                            datasetToVariableMap.put(dataset, variableList);
                        }
                        Map<String,String> infoMap = new HashMap<String, String>();
                        infoMap.put("year", year);
                        infoMap.put("type", survey);
                        infoMap.put("actualname", actualDatasetName);
                        // check if the dataset exists in the information map
                        if (!datasetToInformationMap.containsKey(dataset)) {
                            datasetToInformationMap.put(dataset, infoMap);
                        }

                        NodeList childNodes = conceptNode.getChildNodes();
                        for (int k = 0; k < childNodes.getLength(); k++) {
                            // Variables node
                            Node variablesNode = childNodes.item(k);
                            // Variable nodes
                            NodeList variableNodeList = variablesNode.getChildNodes();
                            for (int j = 0; j < variableNodeList.getLength(); j++) {
                                Node variableNode = variableNodeList.item(j);
                                // the child could be blank space so check
                                // if it is an
                                // element before proceeding
                                if (variableNode instanceof Element) {
                                    String varName = ((Element) variableNode).getFirstChild().getNodeValue();
                                    // add it to the map
                                    datasetToVariableMap.get(dataset).add(varName);
                                }
                            }

                        }
                    }
                    // we now have a map of all the variables so write it
                    // out in case of any server problems
                    //FIXME not used at the moment so commented out
                    //writeDatasetMap(jobID, datasetToVariableMap);

                    Set<Entry<String, List<String>>> entrySet = datasetToVariableMap.entrySet();
                    List<File> fileList = new ArrayList<File>();
                    CSVUtils csvTester = new CSVUtils();
//                    int totalDatasets = datasetToVariableMap.keySet().size();
//                    int currentDataset = 0;
                    for (Entry<String, List<String>> entry : entrySet) {
//                        Map<String, List<String>> readFieldNames;
                        try {
                            String key = entry.getKey();
                            List<String> value = entry.getValue();

                            logger.info("Loading data- " + key + ": " + value);
//                            currentDataset++;
//                            getNumberProcessed().put(
//                                    jobID,
//                                    "Processing dataset " + currentDataset
//                                    + " of " + totalDatasets);
                            Map<String, String> map = datasetToInformationMap.get(key);
                            File csvWriter = csvTester.readFieldNames(key, map.get("type"), map.get("year"),map.get("actualname"),
                                    value);

//                            getNumberProcessed().put(
//                                    jobID,
//                                    "Writing dataset " + currentDataset
//                                    + " of " + totalDatasets);

                            logger.info("Creating file- " + key + ": " + value);
//                            Map<String, String> info = datasetToInformationMap.get(key);
//                            File csvWriter = csvTester.CSVWriter(
//                                    key, readFieldNames,info, jobID);
                            fileList.add(csvWriter);
                        } catch (FileNotFoundException e) {
                            // don't break since the user might want something
                            // TODO errors file
                            logger.error("Problem with thread for job " + jobID
                                    + ": " + e);

                        }
                    }

                    logger.info("Adding files to zip- " + fileList);
//                    getNumberProcessed().put(jobID, "Creating zip file");
                    csvTester.zipWriter(fileList, jobID);
                    BaseResource.getCompletedJobList().put(jobID, true);
                    writeJobState();
                    csvTester.sendEmail(emailAddress, fileName);
                    logger.info("Sent email to " + emailAddress + " for job " + jobID + ", filename " + fileName);

                    //create other file types now
                    //SPSS
                    RServeUtil rserveUtil = new RServeUtil();
                    List<File> spssFiles = new ArrayList<File>();
                    for (File file : fileList) {
                        List<File> createSPSSFiles = rserveUtil.createSPSSFiles(file);
                        for (File spssFile : createSPSSFiles) {
                            spssFiles.add(spssFile);
                        }
                    }

                    csvTester.zipWriter(spssFiles, jobID + "_spss");
                    BaseResource.getSPSSJobList().put(jobID, true);
                    writeSPSSState();
                    //Stata
//                    List<File> stataFiles = new ArrayList<File>();
//                    for (File file : fileList) {
//                        File createStataFiles = rserveUtil.createStataFile(file);
//                        stataFiles.add(createStataFiles);
//                    }
//                    csvTester.zipWriter(stataFiles, jobID + "_stata");
//                    BaseResource.getStataJobList().put(jobID, true);
//                    writeStataState();
                    //remove the request file & the temporary CSV, stata & SPSS files and parent directory
//                    FileUtils.forceDelete(xmlFile);
                    for (File file : spssFiles) {
                        FileUtils.forceDelete(file);
                    }
//                    for (File file : stataFiles) {
//                        FileUtils.forceDelete(file);
//                    }
                    for (File file : fileList) {
                        File parentFile = file.getParentFile();
                        FileUtils.forceDelete(file);
                        FileUtils.forceDelete(parentFile);
                    }
                    logger.info("Current completed jobs are: "
                            + BaseResource.getCompletedJobList().toString());
                } catch (Exception e) {
                    logger.error("Problem with thread for job " + jobID + ": "
                            + e);
                }
            }
        };
        return new Thread(runnable, jobID);

    }

    public static void setFailedJobs(List<String> failedJobs) {
        DownloadFile.failedJobs = failedJobs;
    }

    public static List<String> getFailedJobs() {
        return failedJobs;
    }

    public void setConfig(Configuration config) {
        this.config = config;
    }

    public Configuration getConfig() {
        return config;
    }
}
